#!/usr/bin/env python
# -*- coding: utf-8 -*-

"""
运行Three.js文档结构化爬虫。
这个脚本将读取threejs_structure.html中的完整文档结构，
然后按照层次结构爬取和保存所有文档。
"""

import os
import json
import time
from full_structure_scraper import scrape_by_structure

def main():
    # 配置参数
    base_url = "http://www.yanhuangxueyuan.com/threejs/docs/"
    output_dir = "threejs_docs_structured"
    max_workers = 3  # 并发爬取线程数
    
    # 确保输出目录存在
    os.makedirs(output_dir, exist_ok=True)
    
    # 读取HTML结构文件
    html_file = "threejs_structure.html"
    if not os.path.exists(html_file):
        print(f"错误：无法找到文件 {html_file}")
        print("请确保threejs_structure.html文件包含完整的文档结构")
        return
    
    with open(html_file, 'r', encoding='utf-8') as f:
        html_content = f.read()
    
    print(f"成功读取HTML结构文件，长度：{len(html_content)} 字符")
    print("开始按结构爬取Three.js文档...")
    
    # 运行结构化爬虫
    start_time = time.time()
    docs, structure = scrape_by_structure(
        html_content=html_content, 
        base_url=base_url, 
        output_dir=output_dir, 
        max_workers=max_workers
    )
    end_time = time.time()
    
    # 保存整体结构到单独文件
    structure_file = os.path.join(output_dir, "document_structure.json")
    with open(structure_file, 'w', encoding='utf-8') as f:
        json.dump(structure, f, ensure_ascii=False, indent=2)
    
    # 输出统计信息
    print("\n爬取完成！")
    print(f"总耗时: {end_time - start_time:.2f} 秒")
    print(f"文档总数: {len(docs)}")
    print(f"主分类数: {len(structure)}")
    
    # 统计每个主分类的文档数
    main_categories = {}
    for main_cat in structure:
        main_name = main_cat['name']
        doc_count = sum(len(sub_cat['links']) for sub_cat in main_cat['subcategories'])
        main_categories[main_name] = doc_count
    
    print("\n各主分类文档数量:")
    for cat, count in sorted(main_categories.items()):
        print(f"  {cat}: {count}")
    
    print(f"\n所有文档已保存到: {output_dir}/")
    print("- 每个文档保存为独立的JSON文件，按分类目录组织")
    print("- 完整文档列表保存在: all_docs.json")
    print("- 文档结构保存在: document_structure.json")
    print("- 爬取统计保存在: scrape_stats.json")

if __name__ == "__main__":
    main()
